{
 "cells": [
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T13:42:15.988021Z",
     "start_time": "2024-03-05T13:42:15.545066Z"
    }
   },
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "with open('nas90/data_train_predict/2019-06-03.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:30.371079Z",
     "start_time": "2024-03-05T15:08:30.364064Z"
    }
   },
   "execution_count": 59
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos_adj\n",
      "neg_adj\n",
      "features\n",
      "labels\n",
      "mask\n"
     ]
    }
   ],
   "source": [
    "for key, value in data.items():\n",
    "    print(key)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:30.855908Z",
     "start_time": "2024-03-05T15:08:30.839770Z"
    }
   },
   "execution_count": 60
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([90, 90])"
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['pos_adj'].shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:31.421452Z",
     "start_time": "2024-03-05T15:08:31.404753Z"
    }
   },
   "execution_count": 61
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([90, 90])"
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['neg_adj'].shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:32.252834Z",
     "start_time": "2024-03-05T15:08:32.245603Z"
    }
   },
   "execution_count": 62
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([90, 20, 6])"
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['features'].shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:32.749203Z",
     "start_time": "2024-03-05T15:08:32.736317Z"
    }
   },
   "execution_count": 63
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "torch.Size([90])"
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['labels'].shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:33.899501Z",
     "start_time": "2024-03-05T15:08:33.886690Z"
    }
   },
   "execution_count": 64
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "90"
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data['mask'])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T15:08:34.670536Z",
     "start_time": "2024-03-05T15:08:34.661562Z"
    }
   },
   "execution_count": 65
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "                dt     kdcode     close      high       low      open  \\\n0       2018-12-03  000001.SZ  0.018347  0.018254  0.018982  0.018458   \n1       2018-12-03  000002.SZ  0.047462  0.047567  0.047847  0.048242   \n2       2018-12-03  000063.SZ  0.039212  0.039173  0.039775  0.039120   \n3       2018-12-03  000069.SZ  0.009808  0.009765  0.010105  0.009761   \n4       2018-12-03  000100.SZ  0.002865  0.002836  0.002959  0.002880   \n...            ...        ...       ...       ...       ...       ...   \n252955  2022-12-30  603833.SH  0.231697  0.231878  0.237464  0.232437   \n252956  2022-12-30  603882.SH  0.148368  0.147955  0.149217  0.144688   \n252957  2022-12-30  603899.SH  0.103714  0.103814  0.107208  0.104796   \n252958  2022-12-30  603986.SH  0.195042  0.197178  0.202177  0.199000   \n252959  2022-12-30  603993.SH  0.006731  0.006757  0.007065  0.006861   \n\n              to       vol         label  \n0       0.125469  0.172416  2.080000e-17  \n1       0.172041  0.098539 -2.359527e-02  \n2       0.555836  0.380382 -7.960707e-03  \n3       0.019064  0.045273 -1.144739e-02  \n4       0.020687  0.118699  2.290000e-16  \n...          ...       ...           ...  \n252955  0.016707  0.002001  1.141826e-02  \n252956  0.035236  0.006629  3.019640e-02  \n252957  0.012163  0.003218  5.324255e-02  \n252958  0.061990  0.008789  2.977887e-02  \n252959  0.022141  0.070348  0.000000e+00  \n\n[252960 rows x 9 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>dt</th>\n      <th>kdcode</th>\n      <th>close</th>\n      <th>high</th>\n      <th>low</th>\n      <th>open</th>\n      <th>to</th>\n      <th>vol</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2018-12-03</td>\n      <td>000001.SZ</td>\n      <td>0.018347</td>\n      <td>0.018254</td>\n      <td>0.018982</td>\n      <td>0.018458</td>\n      <td>0.125469</td>\n      <td>0.172416</td>\n      <td>2.080000e-17</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2018-12-03</td>\n      <td>000002.SZ</td>\n      <td>0.047462</td>\n      <td>0.047567</td>\n      <td>0.047847</td>\n      <td>0.048242</td>\n      <td>0.172041</td>\n      <td>0.098539</td>\n      <td>-2.359527e-02</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2018-12-03</td>\n      <td>000063.SZ</td>\n      <td>0.039212</td>\n      <td>0.039173</td>\n      <td>0.039775</td>\n      <td>0.039120</td>\n      <td>0.555836</td>\n      <td>0.380382</td>\n      <td>-7.960707e-03</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2018-12-03</td>\n      <td>000069.SZ</td>\n      <td>0.009808</td>\n      <td>0.009765</td>\n      <td>0.010105</td>\n      <td>0.009761</td>\n      <td>0.019064</td>\n      <td>0.045273</td>\n      <td>-1.144739e-02</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>2018-12-03</td>\n      <td>000100.SZ</td>\n      <td>0.002865</td>\n      <td>0.002836</td>\n      <td>0.002959</td>\n      <td>0.002880</td>\n      <td>0.020687</td>\n      <td>0.118699</td>\n      <td>2.290000e-16</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>252955</th>\n      <td>2022-12-30</td>\n      <td>603833.SH</td>\n      <td>0.231697</td>\n      <td>0.231878</td>\n      <td>0.237464</td>\n      <td>0.232437</td>\n      <td>0.016707</td>\n      <td>0.002001</td>\n      <td>1.141826e-02</td>\n    </tr>\n    <tr>\n      <th>252956</th>\n      <td>2022-12-30</td>\n      <td>603882.SH</td>\n      <td>0.148368</td>\n      <td>0.147955</td>\n      <td>0.149217</td>\n      <td>0.144688</td>\n      <td>0.035236</td>\n      <td>0.006629</td>\n      <td>3.019640e-02</td>\n    </tr>\n    <tr>\n      <th>252957</th>\n      <td>2022-12-30</td>\n      <td>603899.SH</td>\n      <td>0.103714</td>\n      <td>0.103814</td>\n      <td>0.107208</td>\n      <td>0.104796</td>\n      <td>0.012163</td>\n      <td>0.003218</td>\n      <td>5.324255e-02</td>\n    </tr>\n    <tr>\n      <th>252958</th>\n      <td>2022-12-30</td>\n      <td>603986.SH</td>\n      <td>0.195042</td>\n      <td>0.197178</td>\n      <td>0.202177</td>\n      <td>0.199000</td>\n      <td>0.061990</td>\n      <td>0.008789</td>\n      <td>2.977887e-02</td>\n    </tr>\n    <tr>\n      <th>252959</th>\n      <td>2022-12-30</td>\n      <td>603993.SH</td>\n      <td>0.006731</td>\n      <td>0.006757</td>\n      <td>0.007065</td>\n      <td>0.006861</td>\n      <td>0.022141</td>\n      <td>0.070348</td>\n      <td>0.000000e+00</td>\n    </tr>\n  </tbody>\n</table>\n<p>252960 rows × 9 columns</p>\n</div>"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('csi300fealab.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T13:52:34.756300Z",
     "start_time": "2024-03-05T13:52:34.620917Z"
    }
   },
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "             date      open      high       low     close     adjcp    volume  \\\n0      2019-04-09  0.027087  0.026669  0.027101  0.026743  0.026648  0.059211   \n1      2019-04-09  0.042700  0.042523  0.043470  0.042088  0.040852  1.000000   \n2      2019-04-09  0.245494  0.244729  0.249885  0.245061  0.245346  0.011708   \n3      2019-04-09  0.100089  0.098790  0.100804  0.098237  0.091089  0.013539   \n4      2019-04-09  0.145247  0.143911  0.147463  0.143804  0.132388  0.009029   \n...           ...       ...       ...       ...       ...       ...       ...   \n62095  2021-12-31  0.044401  0.044097  0.045394  0.044205  0.041776  0.024842   \n62096  2021-12-31  0.253343  0.251461  0.255592  0.248942  0.249227  0.005497   \n62097  2021-12-31  0.056949  0.056712  0.058084  0.056293  0.056602  0.013543   \n62098  2021-12-31  0.075805  0.075837  0.076842  0.074662  0.074969  0.017545   \n62099  2021-12-31  0.058921  0.058446  0.059273  0.058600  0.056723  0.041601   \n\n      stockcode       pos       neg     ret_1     label  \n0           AAL  0.914460  0.085540  0.983176  1.021315  \n1          AAPL  0.767552  0.232448  0.997001  1.005614  \n2          ADBE  0.500000  0.500000  1.000670  1.009629  \n3           ADI  0.500000  0.500000  0.988813  1.014300  \n4           ADP  0.584781  0.415219  0.995139  1.004634  \n...         ...       ...       ...       ...       ...  \n62095       WBA  0.500000  0.500000  1.003270  1.017255  \n62096      WDAY  0.500000  0.500000  0.987993  0.965371  \n62097       WDC  0.783497  0.216503  0.997552  1.011041  \n62098      WYNN  0.500000  0.500000  0.987115  1.032102  \n62099       XEL  0.500000  0.500000  0.992232  1.003545  \n\n[62100 rows x 12 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>date</th>\n      <th>open</th>\n      <th>high</th>\n      <th>low</th>\n      <th>close</th>\n      <th>adjcp</th>\n      <th>volume</th>\n      <th>stockcode</th>\n      <th>pos</th>\n      <th>neg</th>\n      <th>ret_1</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2019-04-09</td>\n      <td>0.027087</td>\n      <td>0.026669</td>\n      <td>0.027101</td>\n      <td>0.026743</td>\n      <td>0.026648</td>\n      <td>0.059211</td>\n      <td>AAL</td>\n      <td>0.914460</td>\n      <td>0.085540</td>\n      <td>0.983176</td>\n      <td>1.021315</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2019-04-09</td>\n      <td>0.042700</td>\n      <td>0.042523</td>\n      <td>0.043470</td>\n      <td>0.042088</td>\n      <td>0.040852</td>\n      <td>1.000000</td>\n      <td>AAPL</td>\n      <td>0.767552</td>\n      <td>0.232448</td>\n      <td>0.997001</td>\n      <td>1.005614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2019-04-09</td>\n      <td>0.245494</td>\n      <td>0.244729</td>\n      <td>0.249885</td>\n      <td>0.245061</td>\n      <td>0.245346</td>\n      <td>0.011708</td>\n      <td>ADBE</td>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>1.000670</td>\n      <td>1.009629</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2019-04-09</td>\n      <td>0.100089</td>\n      <td>0.098790</td>\n      <td>0.100804</td>\n      <td>0.098237</td>\n      <td>0.091089</td>\n      <td>0.013539</td>\n      <td>ADI</td>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>0.988813</td>\n      <td>1.014300</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>2019-04-09</td>\n      <td>0.145247</td>\n      <td>0.143911</td>\n      <td>0.147463</td>\n      <td>0.143804</td>\n      <td>0.132388</td>\n      <td>0.009029</td>\n      <td>ADP</td>\n      <td>0.584781</td>\n      <td>0.415219</td>\n      <td>0.995139</td>\n      <td>1.004634</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>62095</th>\n      <td>2021-12-31</td>\n      <td>0.044401</td>\n      <td>0.044097</td>\n      <td>0.045394</td>\n      <td>0.044205</td>\n      <td>0.041776</td>\n      <td>0.024842</td>\n      <td>WBA</td>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>1.003270</td>\n      <td>1.017255</td>\n    </tr>\n    <tr>\n      <th>62096</th>\n      <td>2021-12-31</td>\n      <td>0.253343</td>\n      <td>0.251461</td>\n      <td>0.255592</td>\n      <td>0.248942</td>\n      <td>0.249227</td>\n      <td>0.005497</td>\n      <td>WDAY</td>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>0.987993</td>\n      <td>0.965371</td>\n    </tr>\n    <tr>\n      <th>62097</th>\n      <td>2021-12-31</td>\n      <td>0.056949</td>\n      <td>0.056712</td>\n      <td>0.058084</td>\n      <td>0.056293</td>\n      <td>0.056602</td>\n      <td>0.013543</td>\n      <td>WDC</td>\n      <td>0.783497</td>\n      <td>0.216503</td>\n      <td>0.997552</td>\n      <td>1.011041</td>\n    </tr>\n    <tr>\n      <th>62098</th>\n      <td>2021-12-31</td>\n      <td>0.075805</td>\n      <td>0.075837</td>\n      <td>0.076842</td>\n      <td>0.074662</td>\n      <td>0.074969</td>\n      <td>0.017545</td>\n      <td>WYNN</td>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>0.987115</td>\n      <td>1.032102</td>\n    </tr>\n    <tr>\n      <th>62099</th>\n      <td>2021-12-31</td>\n      <td>0.058921</td>\n      <td>0.058446</td>\n      <td>0.059273</td>\n      <td>0.058600</td>\n      <td>0.056723</td>\n      <td>0.041601</td>\n      <td>XEL</td>\n      <td>0.500000</td>\n      <td>0.500000</td>\n      <td>0.992232</td>\n      <td>1.003545</td>\n    </tr>\n  </tbody>\n</table>\n<p>62100 rows × 12 columns</p>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('nas90fealab.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-05T13:53:10.997577Z",
     "start_time": "2024-03-05T13:53:10.968304Z"
    }
   },
   "execution_count": 19
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"E:/baselines/HS300.csv\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T02:05:41.338346Z",
     "start_time": "2024-03-06T02:05:41.178881Z"
    }
   },
   "execution_count": 107
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df['dt'] = pd.to_datetime(df['dt'])\n",
    "grouped_df = df.groupby('kdcode')\n",
    "l = df['dt'].nunique()\n",
    "def filter_groups(group):\n",
    "    return len(group) == l\n",
    "df = grouped_df.filter(filter_groups)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T02:05:42.321412Z",
     "start_time": "2024-03-06T02:05:42.213683Z"
    }
   },
   "execution_count": 108
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "140"
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['kdcode'].nunique()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T02:05:43.232360Z",
     "start_time": "2024-03-06T02:05:43.220322Z"
    }
   },
   "execution_count": 109
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "stock = \"hs300\""
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T02:05:44.765054Z",
     "start_time": "2024-03-06T02:05:44.758941Z"
    }
   },
   "execution_count": 110
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df['ret_1'] = df['close'] / df['close'].shift(1)\n",
    "df['ret_1'] = df['ret_1'].fillna(1)\n",
    "df['label'] = df['ret_1'].shift(-1)\n",
    "df['label'] = df['label'].fillna(1)\n",
    "df.loc[df['dt'] == '2018-01-02', 'ret_1'] = 1\n",
    "df.loc[df['dt'] == '2023-12-28', 'label'] = 1"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T02:05:45.838315Z",
     "start_time": "2024-03-06T02:05:45.814967Z"
    }
   },
   "execution_count": 111
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "           kdcode         dt     close      open      high       low  \\\n0       000001.SZ 2018-01-02  0.004865  0.005158  0.005301  0.005360   \n1       000001.SZ 2018-01-03  0.004723  0.005305  0.005274  0.005312   \n2       000001.SZ 2018-01-04  0.004692  0.005147  0.005088  0.005284   \n3       000001.SZ 2018-01-05  0.004712  0.005104  0.005080  0.005292   \n4       000001.SZ 2018-01-08  0.004581  0.005120  0.005057  0.005175   \n...           ...        ...       ...       ...       ...       ...   \n203975  603993.SH 2023-12-25  0.001492  0.001893  0.001884  0.001960   \n203976  603993.SH 2023-12-26  0.001488  0.001905  0.001880  0.001968   \n203977  603993.SH 2023-12-27  0.001550  0.001897  0.001948  0.001976   \n203978  603993.SH 2023-12-28  0.001581  0.001975  0.001998  0.002052   \n203979  603993.SH 2023-12-29  0.001596  0.001982  0.001986  0.002064   \n\n        turnover    volume     ret_1     label  \n0       0.114687  0.050592  1.000000  0.972993  \n1       0.160845  0.072002  0.972993  0.993998  \n2       0.098547  0.045073  0.993998  1.003774  \n3       0.064370  0.029416  1.003774  0.974436  \n4       0.112662  0.052464  0.974436  1.009259  \n...          ...       ...       ...       ...  \n203975  0.014998  0.018500  1.006122  0.997972  \n203976  0.011185  0.013779  0.997972  1.032520  \n203977  0.030747  0.036935  1.032520  1.015748  \n203978  0.033990  0.039717  1.015748  1.000000  \n203979  0.020546  0.023996  1.007752  1.000000  \n\n[203980 rows x 10 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>kdcode</th>\n      <th>dt</th>\n      <th>close</th>\n      <th>open</th>\n      <th>high</th>\n      <th>low</th>\n      <th>turnover</th>\n      <th>volume</th>\n      <th>ret_1</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>000001.SZ</td>\n      <td>2018-01-02</td>\n      <td>0.004865</td>\n      <td>0.005158</td>\n      <td>0.005301</td>\n      <td>0.005360</td>\n      <td>0.114687</td>\n      <td>0.050592</td>\n      <td>1.000000</td>\n      <td>0.972993</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>000001.SZ</td>\n      <td>2018-01-03</td>\n      <td>0.004723</td>\n      <td>0.005305</td>\n      <td>0.005274</td>\n      <td>0.005312</td>\n      <td>0.160845</td>\n      <td>0.072002</td>\n      <td>0.972993</td>\n      <td>0.993998</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>000001.SZ</td>\n      <td>2018-01-04</td>\n      <td>0.004692</td>\n      <td>0.005147</td>\n      <td>0.005088</td>\n      <td>0.005284</td>\n      <td>0.098547</td>\n      <td>0.045073</td>\n      <td>0.993998</td>\n      <td>1.003774</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>000001.SZ</td>\n      <td>2018-01-05</td>\n      <td>0.004712</td>\n      <td>0.005104</td>\n      <td>0.005080</td>\n      <td>0.005292</td>\n      <td>0.064370</td>\n      <td>0.029416</td>\n      <td>1.003774</td>\n      <td>0.974436</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>000001.SZ</td>\n      <td>2018-01-08</td>\n      <td>0.004581</td>\n      <td>0.005120</td>\n      <td>0.005057</td>\n      <td>0.005175</td>\n      <td>0.112662</td>\n      <td>0.052464</td>\n      <td>0.974436</td>\n      <td>1.009259</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>203975</th>\n      <td>603993.SH</td>\n      <td>2023-12-25</td>\n      <td>0.001492</td>\n      <td>0.001893</td>\n      <td>0.001884</td>\n      <td>0.001960</td>\n      <td>0.014998</td>\n      <td>0.018500</td>\n      <td>1.006122</td>\n      <td>0.997972</td>\n    </tr>\n    <tr>\n      <th>203976</th>\n      <td>603993.SH</td>\n      <td>2023-12-26</td>\n      <td>0.001488</td>\n      <td>0.001905</td>\n      <td>0.001880</td>\n      <td>0.001968</td>\n      <td>0.011185</td>\n      <td>0.013779</td>\n      <td>0.997972</td>\n      <td>1.032520</td>\n    </tr>\n    <tr>\n      <th>203977</th>\n      <td>603993.SH</td>\n      <td>2023-12-27</td>\n      <td>0.001550</td>\n      <td>0.001897</td>\n      <td>0.001948</td>\n      <td>0.001976</td>\n      <td>0.030747</td>\n      <td>0.036935</td>\n      <td>1.032520</td>\n      <td>1.015748</td>\n    </tr>\n    <tr>\n      <th>203978</th>\n      <td>603993.SH</td>\n      <td>2023-12-28</td>\n      <td>0.001581</td>\n      <td>0.001975</td>\n      <td>0.001998</td>\n      <td>0.002052</td>\n      <td>0.033990</td>\n      <td>0.039717</td>\n      <td>1.015748</td>\n      <td>1.000000</td>\n    </tr>\n    <tr>\n      <th>203979</th>\n      <td>603993.SH</td>\n      <td>2023-12-29</td>\n      <td>0.001596</td>\n      <td>0.001982</td>\n      <td>0.001986</td>\n      <td>0.002064</td>\n      <td>0.020546</td>\n      <td>0.023996</td>\n      <td>1.007752</td>\n      <td>1.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>203980 rows × 10 columns</p>\n</div>"
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "# cols_to_normalize = ['close', 'open', 'high', 'low', 'volume']\n",
    "cols_to_normalize = ['close', 'open', 'high', 'low', 'volume', 'turnover']\n",
    "df[cols_to_normalize] = scaler.fit_transform(df[cols_to_normalize])\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T02:05:46.399519Z",
     "start_time": "2024-03-06T02:05:46.364948Z"
    }
   },
   "execution_count": 112
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "           kdcode         dt     close      open      high       low  \\\n0       000001.SZ 2018-01-02  0.004865  0.005158  0.005301  0.005360   \n1       000002.SZ 2018-01-02  0.012119  0.012152  0.012554  0.012656   \n2       000063.SZ 2018-01-02  0.013339  0.014096  0.013928  0.013996   \n3       000069.SZ 2018-01-02  0.003019  0.003284  0.003482  0.003421   \n4       000100.SZ 2018-01-02  0.001100  0.001499  0.001499  0.001561   \n...           ...        ...       ...       ...       ...       ...   \n203975  601989.SH 2023-12-29  0.001185  0.001569  0.001579  0.001630   \n203976  601998.SH 2023-12-29  0.001631  0.002052  0.002032  0.002125   \n203977  603799.SH 2023-12-29  0.012262  0.012635  0.012611  0.013091   \n203978  603833.SH 2023-12-29  0.026370  0.026658  0.026656  0.027610   \n203979  603993.SH 2023-12-29  0.001596  0.001982  0.001986  0.002064   \n\n        turnover    volume     ret_1      label  \n0       0.114687  0.050592  1.000000   0.972993  \n1       0.089070  0.016611  1.000000   0.992936  \n2       0.128280  0.021865  1.000000   1.026868  \n3       0.029534  0.020081  1.000000   1.005618  \n4       0.013984  0.021678  1.000000   1.020460  \n...          ...       ...       ...        ...  \n203975  0.015134  0.022240  1.017241   1.530266  \n203976  0.004988  0.005695  0.994361  15.306238  \n203977  0.031555  0.005800  1.005496   3.689645  \n203978  0.008394  0.000732  1.008256   0.099698  \n203979  0.020546  0.023996  1.007752   1.000000  \n\n[203980 rows x 10 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>kdcode</th>\n      <th>dt</th>\n      <th>close</th>\n      <th>open</th>\n      <th>high</th>\n      <th>low</th>\n      <th>turnover</th>\n      <th>volume</th>\n      <th>ret_1</th>\n      <th>label</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>000001.SZ</td>\n      <td>2018-01-02</td>\n      <td>0.004865</td>\n      <td>0.005158</td>\n      <td>0.005301</td>\n      <td>0.005360</td>\n      <td>0.114687</td>\n      <td>0.050592</td>\n      <td>1.000000</td>\n      <td>0.972993</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>000002.SZ</td>\n      <td>2018-01-02</td>\n      <td>0.012119</td>\n      <td>0.012152</td>\n      <td>0.012554</td>\n      <td>0.012656</td>\n      <td>0.089070</td>\n      <td>0.016611</td>\n      <td>1.000000</td>\n      <td>0.992936</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>000063.SZ</td>\n      <td>2018-01-02</td>\n      <td>0.013339</td>\n      <td>0.014096</td>\n      <td>0.013928</td>\n      <td>0.013996</td>\n      <td>0.128280</td>\n      <td>0.021865</td>\n      <td>1.000000</td>\n      <td>1.026868</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>000069.SZ</td>\n      <td>2018-01-02</td>\n      <td>0.003019</td>\n      <td>0.003284</td>\n      <td>0.003482</td>\n      <td>0.003421</td>\n      <td>0.029534</td>\n      <td>0.020081</td>\n      <td>1.000000</td>\n      <td>1.005618</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>000100.SZ</td>\n      <td>2018-01-02</td>\n      <td>0.001100</td>\n      <td>0.001499</td>\n      <td>0.001499</td>\n      <td>0.001561</td>\n      <td>0.013984</td>\n      <td>0.021678</td>\n      <td>1.000000</td>\n      <td>1.020460</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>203975</th>\n      <td>601989.SH</td>\n      <td>2023-12-29</td>\n      <td>0.001185</td>\n      <td>0.001569</td>\n      <td>0.001579</td>\n      <td>0.001630</td>\n      <td>0.015134</td>\n      <td>0.022240</td>\n      <td>1.017241</td>\n      <td>1.530266</td>\n    </tr>\n    <tr>\n      <th>203976</th>\n      <td>601998.SH</td>\n      <td>2023-12-29</td>\n      <td>0.001631</td>\n      <td>0.002052</td>\n      <td>0.002032</td>\n      <td>0.002125</td>\n      <td>0.004988</td>\n      <td>0.005695</td>\n      <td>0.994361</td>\n      <td>15.306238</td>\n    </tr>\n    <tr>\n      <th>203977</th>\n      <td>603799.SH</td>\n      <td>2023-12-29</td>\n      <td>0.012262</td>\n      <td>0.012635</td>\n      <td>0.012611</td>\n      <td>0.013091</td>\n      <td>0.031555</td>\n      <td>0.005800</td>\n      <td>1.005496</td>\n      <td>3.689645</td>\n    </tr>\n    <tr>\n      <th>203978</th>\n      <td>603833.SH</td>\n      <td>2023-12-29</td>\n      <td>0.026370</td>\n      <td>0.026658</td>\n      <td>0.026656</td>\n      <td>0.027610</td>\n      <td>0.008394</td>\n      <td>0.000732</td>\n      <td>1.008256</td>\n      <td>0.099698</td>\n    </tr>\n    <tr>\n      <th>203979</th>\n      <td>603993.SH</td>\n      <td>2023-12-29</td>\n      <td>0.001596</td>\n      <td>0.001982</td>\n      <td>0.001986</td>\n      <td>0.002064</td>\n      <td>0.020546</td>\n      <td>0.023996</td>\n      <td>1.007752</td>\n      <td>1.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>203980 rows × 10 columns</p>\n</div>"
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.sort_values(by=['dt', 'kdcode'])\n",
    "df = df.reset_index(drop=True)\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T00:49:43.812915Z",
     "start_time": "2024-03-06T00:49:43.729488Z"
    }
   },
   "execution_count": 90
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "<DatetimeArray>\n['2023-01-03 00:00:00', '2023-01-04 00:00:00', '2023-01-05 00:00:00',\n '2023-01-06 00:00:00', '2023-01-09 00:00:00', '2023-01-10 00:00:00',\n '2023-01-11 00:00:00', '2023-01-12 00:00:00', '2023-01-13 00:00:00',\n '2023-01-16 00:00:00',\n ...\n '2023-12-18 00:00:00', '2023-12-19 00:00:00', '2023-12-20 00:00:00',\n '2023-12-21 00:00:00', '2023-12-22 00:00:00', '2023-12-25 00:00:00',\n '2023-12-26 00:00:00', '2023-12-27 00:00:00', '2023-12-28 00:00:00',\n '2023-12-29 00:00:00']\nLength: 242, dtype: datetime64[ns]"
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['dt']>'2023-01-01']['dt'].unique()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T00:49:47.141181Z",
     "start_time": "2024-03-06T00:49:47.119375Z"
    }
   },
   "execution_count": 91
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "df.to_pickle(f\"{stock}fealab.pkl\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-03-06T00:49:48.470678Z",
     "start_time": "2024-03-06T00:49:48.444429Z"
    }
   },
   "execution_count": 92
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.16 ('qlib': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "87496c78f2ce0f880180f5f21e88f5b0eed8306ea5075eae3f7b1cde61f671d8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
